import requests
import sqlite3

from bs4 import BeautifulSoup


conn = sqlite3.connect('w3ctech.db')
c = conn.cursor()


def get_header(user_agent=None, accept=None):
    """return an header,type is dict"""
    if not user_agent:
        user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0'
    if not accept:
        accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    return {
        'User-Agent': user_agent,
        'Accept': accept
    }


def get_article(num):
    url = 'https://www.w3ctech.com/topic/' + str(num)
    header = get_header()
    html = requests.get(url, headers=header)
    if html.status_code==200 and html.content.find('errmsg')==-1:
        bs = BeautifulSoup(html.content)
        title = bs.find(
            'div', attrs={
                'class': 'topic_info clearfix'}).find('h1').text
        create_time = bs.find('time',attrs={'class':'topic_date'}).text
        content = bs.find('div',attrs={'class':'callout'}).text
        c.execute(
            "INSERT INTO article ('Id','Title','CreateTime','Content') VALUES (?,?,?,?)",
            (num, title, create_time, content))
        conn.commit()
    else:
        print str(num)+'does not exist'

for i in range(1, 2062):
    c.execute('select count(Id)from article where Id = ?',(i,))
    if c.fetchall()[0][0]==0:
        get_article(i)
conn.close()
